

import csv
import time
import random
import logging
import os
from datetime import datetime
from urllib.parse import urljoin, urlparse
import re
import json

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException, WebDriverException
from selenium.webdriver.common.action_chains import ActionChains

print(" 2207124003 许熔婷 220706 大数据管理与应用")

class TaobaoSpiderTargeted:
    def __init__(self, chromedriver_path=r"D:\chromeDriver\chromedriver-win64\chromedriver.exe"):
        """
        初始化淘宝爬虫 - 针对已登录页面优化版

        Args:
            chromedriver_path (str): ChromeDriver可执行文件路径
        """
        self.chromedriver_path = chromedriver_path
        self.driver = None
        self.wait = None
        self.data = []

        # 设置日志
        self.setup_logging()

        # 初始化浏览器
        self.setup_driver()

    def setup_logging(self):
        """设置日志配置"""
        logging.basicConfig(
            level=logging.INFO,
            format='%(asctime)s - %(levelname)s - %(message)s',
            handlers=[
                logging.FileHandler('taobao_spider_targeted.log', encoding='utf-8'),
                logging.StreamHandler()
            ]
        )
        self.logger = logging.getLogger(__name__)

    def setup_driver(self):
        """设置Chrome浏览器驱动"""
        try:
            chrome_options = Options()

            # 基础设置
            chrome_options.add_argument('--no-sandbox')
            chrome_options.add_argument('--disable-dev-shm-usage')
            chrome_options.add_argument('--disable-blink-features=AutomationControlled')
            chrome_options.add_experimental_option("excludeSwitches", ["enable-automation"])
            chrome_options.add_experimental_option('useAutomationExtension', False)

            # 设置窗口大小
            chrome_options.add_argument('--window-size=1920,1080')

            # 设置用户代理
            user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
            chrome_options.add_argument(f'--user-agent={user_agent}')

            # 创建服务对象
            service = Service(self.chromedriver_path)

            # 创建驱动实例
            self.driver = webdriver.Chrome(service=service, options=chrome_options)

            # 反检测脚本
            self.driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")

            # 设置等待
            self.wait = WebDriverWait(self.driver, 15)

            self.logger.info("Chrome浏览器驱动初始化成功")

        except Exception as e:
            self.logger.error(f"初始化Chrome驱动失败: {e}")
            raise

    def random_delay(self, min_delay=2, max_delay=4):
        """随机延迟"""
        delay = random.uniform(min_delay, max_delay)
        time.sleep(delay)

    def safe_find_element(self, by, value, timeout=10):
        """安全查找元素"""
        try:
            element = WebDriverWait(self.driver, timeout).until(
                EC.presence_of_element_located((by, value))
            )
            return element
        except TimeoutException:
            return None

    def safe_find_elements(self, by, value, timeout=10):
        """安全查找多个元素"""
        try:
            elements = WebDriverWait(self.driver, timeout).until(
                EC.presence_of_all_elements_located((by, value))
            )
            return elements
        except TimeoutException:
            return []

    def get_text_safe(self, element, default=""):
        """安全获取元素文本"""
        try:
            return element.text.strip() if element else default
        except Exception:
            return default

    def get_attribute_safe(self, element, attribute, default=""):
        """安全获取元素属性"""
        try:
            return element.get_attribute(attribute) if element else default
        except Exception:
            return default

    def wait_for_manual_login(self):
        """等待用户手动登录并确认"""
        print("\n" + "=" * 60)
        print("请按照以下步骤操作：")
        print("1. 在打开的浏览器窗口中登录您的淘宝账号")
        print("2. 确保看到类似截图中的已登录页面")
        print("3. 登录完成后，在此控制台按回车键继续")
        print("=" * 60)

        input("\n登录完成后请按回车键继续...")
        self.logger.info("用户确认已登录，继续执行爬虫")

    def crawl_level1_categories(self, start_url="https://www.taobao.com"):
        self.logger.info("开始爬取一级分类页面...")
        level1_links = []

        try:
            self.driver.get(start_url)
            self.random_delay(3, 5)
            # 等待用户确认登录
            self.wait_for_manual_login()
            category_selectors = [
                # 左侧分类导航
                ".categories-list a",
                ".category-list a",
                ".categorys-list a",
                ".service-bd a",

                # 主要分类链接
                "a[href*='category.taobao.com']",
                "a[href*='list.taobao.com']",
                "a[href*='search']",

                # 顶部导航分类
                ".site-nav-bd a",
                ".nav-item a"
            ]

            # 预定义的主要分类
            predefined_categories = [
                {"name": "国家补贴", "search_term": "国家补贴"},
                {"name": "电脑", "search_term": "电脑"},
                {"name": "家电", "search_term": "家电"},
                {"name": "工业品", "search_term": "工业品"},
                {"name": "家具", "search_term": "家具"},
                {"name": "女装", "search_term": "女装"},
                {"name": "女鞋", "search_term": "女鞋"},
                {"name": "汽车", "search_term": "汽车"},
                {"name": "食品", "search_term": "食品"},
                {"name": "母婴", "search_term": "母婴"},
                {"name": "美妆", "search_term": "美妆"}
            ]

            # 尝试从页面获取分类链接
            for selector in category_selectors:
                try:
                    elements = self.driver.find_elements(By.CSS_SELECTOR, selector)
                    self.logger.info(f"选择器 {selector} 找到 {len(elements)} 个元素")

                    for element in elements[:10]:
                        try:
                            href = self.get_attribute_safe(element, "href")
                            text = self.get_text_safe(element)

                            if (href and text and
                                    len(text.strip()) > 0 and len(text.strip()) < 20 and
                                    ("taobao.com" in href or "tmall.com" in href) and
                                    "javascript:" not in href):
                                level1_links.append({
                                    'url': href,
                                    'title': text.strip(),
                                    'level': 1,
                                    'parent_url': start_url,
                                    'source': 'page_extraction'
                                })
                                self.logger.info(f"提取到分类: {text.strip()}")

                        except Exception as e:
                            continue

                    if len(level1_links) >= 8:
                        break

                except Exception as e:
                    self.logger.warning(f"选择器 {selector} 失败: {e}")
                    continue

            # 如果页面提取的链接不够，使用预定义分类
            if len(level1_links) < 5:
                self.logger.info("页面提取链接不足，使用预定义分类")
                for category in predefined_categories:
                    search_url = f"https://s.taobao.com/search?q={category['search_term']}"
                    level1_links.append({
                        'url': search_url,
                        'title': category['name'],
                        'level': 1,
                        'parent_url': start_url,
                        'source': 'predefined'
                    })

            self.logger.info(f"一级分类爬取完成，共找到 {len(level1_links)} 个分类")
            return level1_links[:10]  # 限制数量

        except Exception as e:
            self.logger.error(f"爬取一级分类失败: {e}")
            return []

    def crawl_level2_products(self, level1_links, max_products=10):
        """
        爬取二级页面商品列表 - 针对搜索结果页面优化

        Args:
            level1_links (list): 一级分类链接列表
            max_products (int): 每个分类最大爬取商品数

        Returns:
            list: 商品链接列表
        """
        self.logger.info("开始爬取二级页面商品列表...")
        level2_links = []

        for i, level1_link in enumerate(level1_links[:6]):
            try:
                self.logger.info(f"正在处理分类 {i + 1}/{len(level1_links[:6])}: {level1_link['title']}")

                self.driver.get(level1_link['url'])
                self.random_delay(3, 5)

                # 等待页面加载
                try:
                    self.wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
                except:
                    pass

                # 滚动页面加载更多内容
                for scroll in range(3):
                    self.driver.execute_script(
                        "window.scrollTo(0, document.body.scrollHeight/3 * {});".format(scroll + 1))
                    time.sleep(1)

                # 针对淘宝搜索结果页面的商品链接选择器
                product_selectors = [
                    # 搜索结果页面商品链接
                    "a[href*='item.taobao.com/item.htm']",
                    "a[href*='detail.tmall.com/item.htm']",

                    # 商品卡片链接
                    ".item .pic-link",
                    ".item .title a",
                    ".items .item a",

                    # 商品图片和标题链接
                    ".item-pic a",
                    ".item-title a",
                    ".product-iWrap a",

                    # 通用商品链接
                    "a[data-nid]",
                    "a[href*='id=']"
                ]

                found_products = []

                for selector in product_selectors:
                    try:
                        elements = self.driver.find_elements(By.CSS_SELECTOR, selector)
                        self.logger.info(f"选择器 {selector} 找到 {len(elements)} 个商品链接")

                        for element in elements:
                            try:
                                href = self.get_attribute_safe(element, "href")

                                # 验证是否为有效的商品链接
                                if (href and
                                        ("item.taobao.com" in href or "detail.tmall.com" in href) and
                                        ("id=" in href or "item.htm" in href)):

                                    # 获取商品标题
                                    title = self.get_text_safe(element)
                                    if not title:
                                        # 尝试从父元素获取标题
                                        try:
                                            parent = element.find_element(By.XPATH, "./..")
                                            title_elem = parent.find_element(By.CSS_SELECTOR,
                                                                             ".title, .item-title, [class*='title']")
                                            title = self.get_text_safe(title_elem)
                                        except:
                                            title = "商品详情"

                                    # 避免重复
                                    if not any(product['url'] == href for product in found_products):
                                        found_products.append({
                                            'url': href,
                                            'title': title or "商品详情",
                                            'level': 2,
                                            'parent_url': level1_link['url'],
                                            'parent_title': level1_link['title'],
                                            'category': level1_link['title']
                                        })

                                        if len(found_products) >= max_products:
                                            break

                            except Exception as e:
                                continue

                        if len(found_products) >= max_products:
                            break

                    except Exception as e:
                        self.logger.warning(f"选择器 {selector} 失败: {e}")
                        continue

                level2_links.extend(found_products)
                self.logger.info(f"从分类 {level1_link['title']} 找到 {len(found_products)} 个商品")

                # 如果没找到商品，保存页面用于调试
                if len(found_products) == 0:
                    debug_filename = f"debug_category_{i}_{level1_link['title']}.html"
                    try:
                        with open(debug_filename, "w", encoding="utf-8") as f:
                            f.write(self.driver.page_source)
                        self.logger.info(f"已保存调试页面: {debug_filename}")
                    except:
                        pass

            except Exception as e:
                self.logger.error(f"处理分类失败 {level1_link['url']}: {e}")
                continue

        self.logger.info(f"二级页面爬取完成，共找到 {len(level2_links)} 个商品链接")
        return level2_links

    def crawl_level3_details(self, level2_links):
        self.logger.info("开始爬取三级页面商品详情...")
        products_data = []

        for i, product_link in enumerate(level2_links[:30]):
            try:
                self.logger.info(f"正在处理商品 {i + 1}/{len(level2_links[:30])}: {product_link['title'][:30]}...")

                self.driver.get(product_link['url'])
                self.random_delay(3, 5)

                # 等待页面加载
                try:
                    self.wait.until(EC.presence_of_element_located((By.TAG_NAME, "body")))
                except:
                    pass

                # 滚动页面
                self.driver.execute_script("window.scrollTo(0, 500);")
                time.sleep(1)

                # 提取商品信息
                product_data = {
                    'url': product_link['url'],
                    'parent_url': product_link['parent_url'],
                    'parent_category': product_link['category'],
                    'crawl_time': datetime.now().strftime('%Y-%m-%d %H:%M:%S')
                }

                # 商品标题
                title_selectors = [
                    "h1[data-spm='1000983']",
                    ".tb-detail-hd h1",
                    "h1",
                    ".item-title",
                    ".product-title",
                    "[class*='ItemTitle']",
                    "[class*='title']"
                ]

                for selector in title_selectors:
                    title_element = self.safe_find_element(By.CSS_SELECTOR, selector, timeout=3)
                    if title_element:
                        title_text = self.get_text_safe(title_element)
                        if title_text and len(title_text.strip()) > 5:
                            product_data['title'] = title_text.strip()
                            break
                else:
                    product_data['title'] = product_link['title']

                # 商品价格
                price_selectors = [
                    ".tb-rmb-num",
                    ".tm-price-panel .tm-price",
                    ".price",
                    "[class*='price']",
                    ".current-price",
                    ".sale-price"
                ]

                for selector in price_selectors:
                    price_element = self.safe_find_element(By.CSS_SELECTOR, selector, timeout=3)
                    if price_element:
                        price_text = self.get_text_safe(price_element)
                        if price_text:
                            # 提取价格数字
                            price_match = re.search(r'[\d.,]+', price_text.replace('￥', '').replace('¥', ''))
                            if price_match:
                                product_data['price'] = price_match.group()
                                break
                else:
                    product_data['price'] = "未知"

                # 销量信息
                sales_selectors = [
                    ".tb-count",
                    "[class*='sales']",
                    "[class*='sold']",
                    ".deal-cnt"
                ]

                for selector in sales_selectors:
                    sales_element = self.safe_find_element(By.CSS_SELECTOR, selector, timeout=3)
                    if sales_element:
                        sales_text = self.get_text_safe(sales_element)
                        if sales_text:
                            product_data['sales'] = sales_text.strip()
                            break
                else:
                    product_data['sales'] = "未知"

                # 店铺名称
                shop_selectors = [
                    ".tb-seller-name",
                    ".shop-name",
                    "[class*='shop']",
                    ".seller-name"
                ]

                for selector in shop_selectors:
                    shop_element = self.safe_find_element(By.CSS_SELECTOR, selector, timeout=3)
                    if shop_element:
                        shop_text = self.get_text_safe(shop_element)
                        if shop_text:
                            product_data['shop_name'] = shop_text.strip()
                            break
                else:
                    product_data['shop_name'] = "未知"

                # 评价数量
                review_selectors = [
                    ".tb-rate-counter",
                    "[class*='review']",
                    "[class*='comment']",
                    ".rate-num"
                ]

                for selector in review_selectors:
                    review_element = self.safe_find_element(By.CSS_SELECTOR, selector, timeout=3)
                    if review_element:
                        review_text = self.get_text_safe(review_element)
                        if review_text:
                            product_data['reviews'] = review_text.strip()
                            break
                else:
                    product_data['reviews'] = "未知"

                # 商品描述
                desc_selectors = [
                    ".tb-detail-bd",
                    ".item-desc",
                    "[class*='desc']",
                    ".detail-content"
                ]

                for selector in desc_selectors:
                    desc_element = self.safe_find_element(By.CSS_SELECTOR, selector, timeout=3)
                    if desc_element:
                        desc_text = self.get_text_safe(desc_element)
                        if desc_text and len(desc_text.strip()) > 10:
                            product_data['description'] = desc_text.strip()[:200] + "..." if len(
                                desc_text) > 200 else desc_text.strip()
                            break
                else:
                    product_data['description'] = "无描述"

                products_data.append(product_data)
                self.logger.info(f"成功提取商品: {product_data['title'][:30]}...")

            except Exception as e:
                self.logger.error(f"处理商品详情失败 {product_link['url']}: {e}")
                continue

        self.logger.info(f"三级页面爬取完成，共提取 {len(products_data)} 个商品详情")
        return products_data

    def save_to_csv(self, data, filename=None):
        """保存数据到CSV文件"""
        if not data:
            self.logger.warning("没有数据需要保存")
            return None

        if not filename:
            timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
            filename = f"taobao_products_targeted_{timestamp}.csv"

        try:
            fieldnames = [
                'title', 'price', 'sales', 'shop_name', 'reviews',
                'description', 'url', 'parent_url', 'parent_category', 'crawl_time'
            ]

            with open(filename, 'w', newline='', encoding='utf-8-sig') as csvfile:
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                writer.writeheader()

                for item in data:
                    row = {}
                    for field in fieldnames:
                        row[field] = item.get(field, '')
                    writer.writerow(row)

            self.logger.info(f"数据已保存到 {filename}，共 {len(data)} 条记录")
            return filename

        except Exception as e:
            self.logger.error(f"保存CSV文件失败: {e}")
            return None

    def run(self, start_url="https://www.taobao.com", output_file=None):
        """运行完整的爬虫流程"""
        try:
            self.logger.info("开始运行淘宝多级页面爬虫（针对性优化版）...")

            # 第一步：爬取一级分类
            level1_links = self.crawl_level1_categories(start_url)
            if not level1_links:
                self.logger.error("未找到一级分类链接，爬虫终止")
                return None

            # 第二步：爬取二级商品列表
            level2_links = self.crawl_level2_products(level1_links)
            if not level2_links:
                self.logger.error("未找到商品链接，爬虫终止")
                return None

            # 第三步：爬取三级商品详情
            products_data = self.crawl_level3_details(level2_links)
            if not products_data:
                self.logger.error("未提取到商品数据，爬虫终止")
                return None

            # 第四步：保存数据
            output_file = self.save_to_csv(products_data, output_file)

            self.logger.info("爬虫运行完成！")
            return output_file

        except Exception as e:
            self.logger.error(f"爬虫运行失败: {e}")
            return None

    def close(self):
        """关闭浏览器"""
        if self.driver:
            self.driver.quit()
            self.logger.info("浏览器已关闭")


def main():
    """主函数"""
    # ChromeDriver路径
    chromedriver_path = r"D:\chromeDriver\chromedriver-win64\chromedriver.exe"

    print("淘宝多级页面爬虫 - 针对已登录页面优化版")
    print("=" * 60)
    print("注意：此版本专门针对已登录的淘宝页面进行优化")
    print("请确保您能够手动登录淘宝账号")
    print("=" * 60)

    # 创建爬虫实例
    spider = TaobaoSpiderTargeted(chromedriver_path)

    try:
        # 运行爬虫
        output_file = spider.run()

        if output_file:
            print(f"\n✅ 爬虫运行成功！")
            print(f"📁 数据已保存到: {output_file}")
            print(f"📊 请查看CSV文件获取爬取结果")
        else:
            print(f"\n❌ 爬虫运行失败")
            print(f"📋 请查看日志文件 taobao_spider_targeted.log 获取详细信息")
            print(f"🔧 如有调试页面生成，请检查 debug_*.html 文件")

    except KeyboardInterrupt:
        print("\n⏹️ 用户中断爬虫运行")

    except Exception as e:
        print(f"\n💥 爬虫运行出错: {e}")

    finally:
        # 关闭浏览器
        spider.close()


if __name__ == "__main__":
    main()

